<?php
/********************************************************************
 * project: tardb -- tar file database
 * file: rebuild.php
 * author: xtaci
 * email: xtaci@163.com
 * description:
 *    This file is intended for building tar file db for fast access 
 *  contents inside tarballs without untar the tarballs. It will create
 *  2 database files in the directory specified in 
 *  settings.php -- 'filedb' and 'db'.
 *    The 'filedb' file is created to store md5sum of the tarballs and paths
 *  to the tarballs, it is stored as key-value pairs as followings:
 *             
 *  	   md5sum(tarball1)-> path to tarball1
 *         md5sum(tarball2)-> path to tarball2
 *  	   ...
 *  
 *    The 'db' file is built to index all files in tarballs into one big
 *  file, this db contains key-value pairs as followings:
 *  
 *        ...
 *        filename3 -> md5sum(tarball1)*(file offset)*(file size)
 *        filename4 -> md5sum(tarball2)*(file offset)*(file size)
 *        ...
 *  
 *    The value part is a combined string which can easily be exploded into
 *  three parts:
 *      md5sum(tarball1) -- for locating the tarball in filedb;
 *      file offset      -- for fseek the file in tarball;
 *      file size        -- specifies the size of the file.
 *
 * revision: 1.0
 * date: 2009/12/16
 */
require_once('settings.php');

echo "tardb -- tar file database\n\n"; 

// recursively get all tarballs
$files = get_all_files($tar_root);
echo 'ALL TARBALLS IN '.$tar_root."\n";
print_r($files);

/// INDEX TARBALL FILEDB.
$filedb = dba_open($filedbpath, "c", "db4");

if (!$filedb) {
    echo "dba_open failed when openning filedb.\n";
    exit;
}

// exclude files that already been indexed.
for($k = dba_firstkey($filedb); $k != false; $k = dba_nextkey($filedb)) {
    $tarfile = dba_fetch($k, $filedb);
    if (in_array($tarfile, $files)) {
        remove_element($files, $tarfile);
    }
}

echo 'NEW TARBALLS IN ['.$tar_root."]\n";
print_r($files);

/// HASHING NEW TARBALLS
$updatelist = array();
foreach ($files as $tarfile)
{
    echo 'Hashing '.$tarfile.'...';
    $md5ret = exec('md5sum '.$tarfile);
    $fields = explode(' ',$md5ret);
    $md5sum = $fields[0];
    echo '['.$md5sum."]\n";
    // if the tarball does not exists, insert into filedb.
    if (!dba_exists($md5sum, $filedb)) {
        $updatelist[] = $tarfile;
    }
    // force replace the value in case of tarball path changes.
    dba_replace($md5sum, $tarfile, $filedb);
}

dba_sync($filedb);
/// GET TARFILE->MD5SUM HASH, reverse list.
$rev_list = array();
for($k = dba_firstkey($filedb); $k != false; $k = dba_nextkey($filedb)) {
    $tarfile = dba_fetch($k, $filedb);
	$rev_list[$tarfile] = $k;
}

dba_close($filedb);

/// INDEXING TARS
$db = dba_open($dbpath, "c", "db4");

if (!$db) {
    echo "dba_open failed when openning db.\n";
    exit;
}

$patch_array = load_patches();

if (count($patch_array)==0)
{
	// index all new tarballs
	$i = 1;
	$count = count($updatelist);
	echo $count." tarballs need to index.\n";

	foreach ($updatelist as $tarfile)
	{
		//
		$md5sum = $rev_list[$tarfile];
		echo '['.$i.'/'.$count.'] Creating index for '.$tarfile."\n";
		$tarreader = new TarReader($tarfile,$md5sum);
		$tarreader->_createIndex($db);
		dba_sync($db);
		$i++;
	}
}
else
{
	print_r($patch_array);
	// index patches
	$i = 1;
	$count = count($patch_array);
	echo $count." patches need to index.\n";

	foreach ($patch_array as $tarfile)
	{
		$md5sum = $rev_list[$tarfile];
		echo '['.$i.'/'.$count.'] Creating index for '.$tarfile.' ['.$md5sum."]\n";
		$tarreader = new TarReader($tarfile,$md5sum);
		$tarreader->_createIndex($db);
		dba_sync($db);
		$i++;
	}

	// rename tarball.txt to _tarball.txt
	// rename($patchfile, $tar_root.'/_tarballs.txt');
}

// everything is fine, shutdown db and exit;
dba_close($db);

/////////////////////// FUNCTIONS /////////////////////////////////////////
/// read all files including subdirectories
function get_all_files( $path ){
    $list = array();
    foreach( glob( $path . '/*') as $item ){
        if( is_dir( $item ) ){
         $list = array_merge( $list , get_all_files( $item ) );
        }
        else{
            if(substr(strtoupper($item),strlen($item)-4,4) == '.TAR')
                 $list[] = $item;
        }
    }
    return $list;
}

/// strip element from array()
function remove_element(&$arr, $val){
    foreach ($arr as $key => $value) {
        if ($arr[$key] == $val) unset($arr[$key]);
    }
    return $arr = array_values($arr);
}

/// read lines from a file into an array
function load_patches()
{
  $arr = array();
  $handle = @fopen($GLOBALS['patchfile'], "r");
  if ($handle) {
    while (!feof($handle)) {
	  $path = trim(fgets($handle, 4096));
      if($path !='') {
	    if(substr($path,0,1)!='/') {
	 	  $path =  $GLOBALS['tar_root'].'/'.$path;
	    }
        $arr[] = $path;
      }
    }
    fclose($handle);
  }
  return $arr;
}

/// Class for indexing tar
Class TarReader
{
    /**
    * @var string Name of the Tar
    */
    var $_tarname='';

    /**
    * @var file descriptor
    */
    var $_file=0;


    /**
    * @var md5sum of file
    */
    var $_md5sum;

    function TarReader($p_tarname, $md5sum=0)
    {
        $this->_tarname = $p_tarname;
        $this->_md5sum = $md5sum;
    }

    // {{{ destructor
    function _TarReader()
    {
        $this->_close();
    }

    function _createIndex($db)
    {
		if($this->_md5sum =='')
			return false;

        $this->_openRead();
        $offset = 0;
        while (strlen($v_binary_data = $this->_readBlock()) != 0)
        {
            if (!$this->_readHeader($v_binary_data, $v_header))
                return false;
            if ($v_header['filename'] == '') {
                continue;
            }
            if ($v_header['typeflag'] != 0) {
                continue;
            }
            $value = $this->_md5sum.'*'.@ftell($this->_file).'*'.$v_header['size'];
            dba_replace($v_header['filename'], $value, $db);
            $this->_jumpBlock(ceil(($v_header['size']/512)));
        }
        $this->_close();
    }

    // {{{ _readHeader()
    function _readHeader($v_binary_data, &$v_header)
    {
        if (strlen($v_binary_data) != 512) {
            $v_header['filename'] = '';
            $this->_error('Invalid block size : '.strlen($v_binary_data));
            return false;
        }

//        $v_data = unpack("a100filename/a8mode/a8uid/a8gid/a12size/a12mtime/"
//                       ."a8checksum/a1typeflag/a100link/a6magic/a2version/"
//                       ."a32uname/a32gname/a8devmajor/a8devminor",
//                       $v_binary_data);

        // ----- Extract the properties
        $v_header['filename'] = trim(substr($v_binary_data,0,100));
        $v_header['size'] = OctDec(trim(substr($v_binary_data,124,12)));
        $v_header['typeflag'] = substr($v_binary_data,156,1);

        return true;
    }

	// {{{ _openRead()
    function _openRead()
    {      
        $v_filename = $this->_tarname;
        $this->_file = @fopen($v_filename, "rb");

        if ($this->_file == 0) {
            $this->_error('Unable to open in read mode \''.$v_filename.'\'');
            return false;
        }

        return true;
    }
    // }}}

    // {{{ _readBlock()
    function _readBlock()
    {
      return @fread($this->_file, 512);
    }

    // {{{ _jumpBlock()
    function _jumpBlock($p_len)
    {
	  return @fseek($this->_file, ftell($this->_file)+($p_len*512));
    }

    function _close()
    {
		@fclose($this->_file);
		$this->_file = 0;

        return true;
    }

    function _error($p_message)
    {
        // ----- To be completed
        echo $p_message;
    }
}
?>